﻿
namespace WebCommit.Common.Helpers
{
    using System;
    using System.Collections.ObjectModel;
    using System.Linq;
    using System.Text.RegularExpressions;
	using System.Net;
	using System.IO;
	using System.Text;

    /// <summary>
    /// foamliu, 2009/12/27, 页面解析.
    /// HTML代码不需要像程序语言那样经过严格的语法检查, 而且有为数众多的非专业网页编辑人员的存在, 
    /// 因此爬虫的页面解析必须足够宽容, 不能因为小小错误而把许多重要的网页丢弃.
    /// 
    /// 下面的简单实现只提取页面的链接信息.
    /// 它会去:
    /// 1.寻找相应标签并获取其href属性值.
    /// 
    /// 它不会去:
    /// 1.区分静态和动态页面, 更加不会自主构造查询URL.
    /// </summary>
    public class Parser
    {
		public static string ExtractHtml(HttpWebResponse resp,ref string charSet)
		{
			string encoding = resp.ContentEncoding;
			Stream stream;
			if (encoding == "gzip")
				stream = new System.IO.Compression.GZipStream(resp.GetResponseStream(), System.IO.Compression.CompressionMode.Decompress);
			else
				stream = resp.GetResponseStream();

			byte[] buffer = Utility.ReadInstreamIntoMemory(stream);
			string str = Encoding.ASCII.GetString(buffer);

			charSet = Regex.Match(str, @"charset\b\s*=\s*[""']?([^""'>\s]+)[""'>\s]?", RegexOptions.IgnoreCase).Groups[1].Value;
			if (string.IsNullOrEmpty(charSet))
				charSet = Utility.GetAppCfg<string>("DefaultCharSet");
			
			string html = Encoding.GetEncoding(charSet).GetString(buffer);

			return html;
		}

		public static string ExtractTitle(string html)
		{
			string title = Regex.Match(html, @"<title>([^(</)]*)</title>", RegexOptions.IgnoreCase).Groups[1].Value;
			return title;
		}

		public static string ExtractTitle(HttpWebResponse resp,ref string charSet)
		{
			string encoding = resp.ContentEncoding;
			Stream stream;
			if (encoding == "gzip")
				stream = new System.IO.Compression.GZipStream(resp.GetResponseStream(), System.IO.Compression.CompressionMode.Decompress);
			else
				stream = resp.GetResponseStream();

			byte[] buffer = new byte[0];
			//stream.Position = 0;
			//stream.Read(buffer, 0, 1024);
			//string str = Encoding.ASCII.GetString(buffer);

			//charSet = Regex.Match(str, @"charset\b\s*=\s*[""']?([^""'>\s]+)[""'>\s]?", RegexOptions.IgnoreCase).Groups[1].Value;
			//if (string.IsNullOrEmpty(charSet))
			//    charSet = "UTF-8";
			charSet = ExtractCharSet(stream, ref buffer);

			string str2 = Encoding.GetEncoding(charSet).GetString(buffer);
			string title = Regex.Match(str2, @"<title>([^(</)]*)</title>", RegexOptions.IgnoreCase).Groups[1].Value;
			return title;
		}

		private static string ExtractCharSet(Stream stream, ref byte[] buffer)
		{ 
			int size = 1024;
			int offSet= buffer.Length;
			
			byte[] newBuffer = new byte[size];
			int readCount = stream.Read(newBuffer, 0, size);
			buffer = buffer.Concat(newBuffer).ToArray();
			string str = Encoding.ASCII.GetString(buffer);
			string charSet = Regex.Match(str, @"charset\b\s*=\s*[""']?([^""'>\s]+)[""'>\s]?", RegexOptions.IgnoreCase).Groups[1].Value;

			if (!string.IsNullOrEmpty(charSet))
			{
				return charSet;
			}
			if (string.IsNullOrEmpty(charSet) && readCount>0 && readCount==size)
			{
				return ExtractCharSet(stream, ref buffer);
			}
			else// (string.IsNullOrEmpty(charSet) && readCount == 0)
			{
				return Utility.GetAppCfg<string>("DefaultCharSet"); 
			}

		}

        public static string[] ExtractLinks(string baseUri, string html)
        {
            Collection<string> urls = new Collection<string>();

            try
            {
                string strRef = @"(href|HREF|src|SRC)[ ]*=[ ]*[""'][^""'#>]+[""']";
                MatchCollection matches = new Regex(strRef).Matches(html);
                
                foreach (Match match in matches)
                {
                    strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\'', '#', ' ', '>');
                    try
                    {
                        if (IsGoodUri(strRef))
                        {
                            Utility.Normalize(baseUri, ref strRef);
                            urls.Add(strRef);
                        }
                    }
                    catch (Exception)
                    {
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }

            //Logger.Info("Found: " + urls.Count + " ref(s)\r\n");

            return urls.ToArray();
        }


        public static bool IsGoodUri(string strUri)
        {
            if (strUri.ToLower().StartsWith("javascript:"))
                return false;
            return true;
        }
       
    }
}
